package com.xiaotu.spider.douban.pageprocessor;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.Connection.Response;
import org.jsoup.nodes.DataNode;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.nodes.Node;
import org.jsoup.nodes.TextNode;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Scope;
import org.springframework.stereotype.Component;

import com.xiaotu.common.model.TVNotBroadcastModel;
import com.xiaotu.common.redis.RedisClientTemplate;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.GsonUtils;
import com.xiaotu.common.util.KeywordEncode;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.PropertiesUtil;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.StringUtil;
import com.xiaotu.spider.AUrlPageProcessor;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

/**
 * 豆瓣未上映电视剧电影爬取
 */
@Component("DouBanTVNotBrdPageProcessor")
@Scope("prototype")
public class DouBanTVNotBrdPageProcessor extends AUrlPageProcessor
{
	
	private String start_url = "https://www.baidu.com/";
	
	private List<TVNotBroadcastModel> pipeList = new ArrayList<TVNotBroadcastModel>();
	
	@Autowired
	private RedisClientTemplate redisClient;
	
	
	
	@Override
	protected String[] getStartRequestUrls() {
		return new String[]{start_url};
	}

	//链接目标url
  	public static Connection connent(String url) {
      	Connection conn = Jsoup.connect(url)    
                  .userAgent("Mozilla/5.0 (Windows NT 6.1; rv:47.0) Gecko/20100101 Firefox/47.0") // http请求的浏览器设置
                  .timeout(5000)   // http连接时长
                  .maxBodySize(0)
                  .method(Connection.Method.GET);
                  //.cookie("cookie", PropertiesUtil.getProperty("DouBanTVNotBrd.site.Cookie"));  	
      	return conn;
      }	
	
    @Override
    public String getPageDataType(Page page)
    {
        return null;
    }

    @Override
    public void process(Page page)
    {
    	List<JSONObject> targetList = getStarname();
    	
        for (int i = 0; i < targetList.size(); i++)
        {
        	
            Connection conn = connent("https://movie.douban.com/celebrities/search?search_text=" 
        			+ KeywordEncode.encode(targetList.get(i).getString("name")));
			try {
				conn.get();
				Response rs = conn.response();
    			String html = rs.body();
    			if(StringUtil.isNotEmpty(html)){
    				Document documentHtml = Jsoup.parse(html,"utf8");
    				String href = documentHtml.select(".result .pic").select("a").attr("href");
    				if(StringUtils.isNotBlank(href)){
    					this.starRecordProcess(page,href);
    				}
    			}
			} catch (IOException e) {
				e.printStackTrace();
			}
            
        }
    }

    private void starRecordProcess(Page page,String url){
    	
    	Connection conn = connent(url);
    	
    	try {
			conn.get();
			Response rs = conn.response();
			String html = rs.body();
			if(StringUtil.isNotEmpty(html)){
				Document documentHtml = Jsoup.parse(html,"utf8");
				Elements liTagelments = documentHtml.select("#recent_movies").select(".unreleased").select("li");
				for(Element litag :liTagelments){
					
					String tvUrl = litag.select("a").attr("href");
					System.out.println(tvUrl);
					String dateTime = litag.select("span").text();
					if(dateTime.contains("(")){
						dateTime = dateTime.replace("(", "").replace(")", "");
					}
					if(StringUtils.isNotBlank(tvUrl)){
						this.tvRecordProcess(page,tvUrl,dateTime);
					}
				}
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
    	
    }
    List<String> listTvname = new ArrayList<String>();
    private void tvRecordProcess(Page page,String url,String dateTime)
    {
    	Map<String,Object> tvMap = new HashMap<String,Object>();
    	String urlId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER,url);
    	
    	Connection conn = connent(url);
    	
    	try {
			conn.get();
			Response rs = conn.response();
			String html = rs.body();
			if(StringUtil.isNotEmpty(html)){
				Document documentHtml = Jsoup.parse(html,"utf8");
				String tvname = documentHtml.select("#content").select("span[property=v:itemreviewed]").text();
				
				if(StringUtil.isNotEmpty(tvname)){
		    		tvname = tvname.trim();
		    		if (redisClient.hexists(Constants.DataRedisKey.Star_Info_Snap_All.toString(), tvname)){
		    			return;
		    		}
		    		if(listTvname.contains(tvname)){
		    			return;
		    		}else{
		    			listTvname.add(tvname);
		    			tvMap.put("tvname"+urlId, tvname);
		    		}
		    	}
				
				String releasetime = null;
		    	if(StringUtil.isNotEmpty(dateTime)){
					releasetime = dateTime.trim();
					tvMap.put("releasetime"+urlId, releasetime);
		    	}
				
		    	Elements spanPl = documentHtml.select(".subject.clearfix").select("#info").select("span[class=pl]");
		    	
		    	for(Element spanPlElement : spanPl){
		    		
		    		String prefix = spanPlElement.text();
			    	
		    		if(prefix.contains("导演")){
			    		String directorStr = ""; 
			    		Elements directors = spanPlElement.siblingElements().select("span[class=attrs]").select("a");
			    		for(Element director : directors){
			    			directorStr+="/"+director.text();
			    		}
			    		if(StringUtils.isNotBlank(directorStr)){
			    			directorStr = directorStr.substring(1);
			    		}
			    		tvMap.put("director"+urlId, directorStr);
			    		continue;
			    	}
		    		
		    		if(prefix.contains("编剧")){
			    		String writerStr = ""; 
			    		Elements writers = spanPlElement.siblingElements().select("span[class=attrs]").select("a");
			    		for(Element writer : writers){
			    			writerStr+="/"+writer.text();
			    		}
			    		if(StringUtils.isNotBlank(writerStr)){
			    			writerStr = writerStr.substring(1);
			    		}
			    		tvMap.put("writer"+urlId, writerStr);
			    		continue;
			    	}
		    		
			    	if(prefix.contains("类型")){
			    		String labelStr = ""; 
			    		Elements labels = spanPlElement.siblingElements().select("span[property=v:genre]");
			    		for(Element label : labels){
			    			labelStr+="/"+label.text();
			    		}
			    		if(StringUtils.isNotBlank(labelStr)){
			    			labelStr = labelStr.substring(1);
			    		}
			    		tvMap.put("showtype"+urlId, labelStr);
			    		continue;
			    	}
			    	
			    	if(prefix.contains("制片国家")){
			    		String areaStr = "";
			    		List<Node> allNodes = documentHtml.select("#info").get(0).childNodes();
			    		for(int i = 0;i<allNodes.size();i++){
			    			Node node = allNodes.get(i);
			    			if(node.toString().contains("制片国家")){
			    				areaStr = allNodes.get(i+1).toString();
			    				break;
			    			}
			    		}
			    		areaStr = areaStr.trim();
			    		tvMap.put("area"+urlId, areaStr);
			    		continue;
			    	}
			    	
			    	if(prefix.contains("语言")){
			    		String languageStr = "";
			    		List<Node> allNodes = documentHtml.select("#info").get(0).childNodes();
			    		for(int i = 0;i<allNodes.size();i++){
			    			Node node = allNodes.get(i);
			    			if(node.toString().contains("语言")){
			    				languageStr = allNodes.get(i+1).toString();
			    				break;
			    			}
			    		}
			    		languageStr = languageStr.trim();
			    		tvMap.put("language"+urlId, languageStr);
			    		continue;
			    	}
			    	
			    	if(prefix.contains("集数")){
			    		String setsStr = "";
			    		List<Node> allNodes = documentHtml.select("#info").get(0).childNodes();
			    		for(int i = 0;i<allNodes.size();i++){
			    			Node node = allNodes.get(i);
			    			if(node.toString().contains("集数")){
			    				setsStr = allNodes.get(i+1).toString();
			    				break;
			    			}
			    		}
			    		setsStr = setsStr.trim();
			    		tvMap.put("sets"+urlId, setsStr);
			    		continue;
			    	}
			    	
			    	
			    	if(prefix.contains("单集片长")){
			    		String timelengthStr = "";
			    		List<Node> allNodes = documentHtml.select("#info").get(0).childNodes();
			    		for(int i = 0;i<allNodes.size();i++){
			    			Node node = allNodes.get(i);
			    			if(node.toString().contains("单集片长")){
			    				timelengthStr = allNodes.get(i+1).toString();
			    				break;
			    			}
			    		}
			    		timelengthStr = timelengthStr.trim().replace("分钟", "");
			    		tvMap.put("timelength"+urlId, timelengthStr);
			    		continue;
			    	}
			    	
			    	if(prefix.contains("又名")){
			    		String aliasStr = "";
			    		List<Node> allNodes = documentHtml.select("#info").get(0).childNodes();
			    		for(int i = 0;i<allNodes.size();i++){
			    			Node node = allNodes.get(i);
			    			if(node.toString().contains("又名")){
			    				aliasStr = allNodes.get(i+1).toString();
			    				break;
			    			}
			    		}
			    		aliasStr = aliasStr.trim().replace(" ", "");
			    		tvMap.put("alias"+urlId, aliasStr);
			    		continue;
			    	}
			    	
			    	
		    	}
		    	
		    	//简介
		    	String introduction = documentHtml.select(".related-info").select("#link-report").select("span[class=all hidden]").text();
		    	
		    	if(StringUtils.isBlank(introduction)){
		    		introduction = documentHtml.select(".related-info").select("#link-report").select("span[property=v:summary]").text();
		    	}
		    	if(StringUtils.isNotBlank(introduction)){
		    		introduction = introduction.replace("　", "").trim();
		    	}
		    	tvMap.put("introduction"+urlId, introduction);
		    
		    	
		    	//获取主要演员
		    	this.commentProcess(page,"https://movie.douban.com/subject/"+urlId+"/celebrities",tvMap);
		    	
			}
			
		} catch (IOException e) {
			e.printStackTrace();
		}
    	
    }

    private void commentProcess(Page page,String url, Map<String, Object> tvMap)
    {
    	String[] actorArr = null;
    	String[] actorRoleArr = null;
    	String urlId = RegexUtils.regexSearch(RegexUtils.REGEX_INTEGER,url);
    	
    	Connection conn = connent(url);
    	
    	try {
			conn.get();
			Response rs = conn.response();
			String html = rs.body();
			if(StringUtil.isNotEmpty(html)){
				Document documentHtml = Jsoup.parse(html,"utf8");
				Elements h2Tagelments = documentHtml.select(".article").select("#celebrities").select("div[class=list-wrapper]").select("h2");
				
				for(int i=0;i<h2Tagelments.size();i++){
					if(h2Tagelments.get(i).toString().contains("演员 Cast")){
						
						Elements infoTags = h2Tagelments.get(i).siblingElements().select("ul[class=celebrities-list  __multiline]").select("li").select("div[class=info]");
						actorArr = new String[infoTags.size()];
						actorRoleArr = new String[infoTags.size()];
						for(int j = 0;j<infoTags.size();j++){
							
							Element info = infoTags.get(j);
							String actorName = info.select("span[class=name]").select("a").text();
							if(isContainChinese(actorName)){
								actorName =RegexUtils.regexSearch(RegexUtils.REGEX_CH,actorName);
								actorArr[j]=actorName;
					    	}else{
					    		actorArr[j]=actorName;
					    	}
							String actorRole = info.select("span[class=role]").text();
							if(StringUtil.isNotEmpty(actorRole) && actorRole.contains("饰")){
								actorRole =actorRole.substring(actorRole.lastIndexOf("饰")+1,actorRole.lastIndexOf(")"));
								actorRoleArr[j]=actorRole;
							}else{
								actorRoleArr[j]=null;
							}
							
						}
					}
				}
				
				
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
    	
    	//缓存爬取过得未播剧名称
    	redisClient.hset(Constants.DataRedisKey.Star_Info_Snap_All.toString(), tvMap.get("tvname"+urlId).toString(), DateUtils
                .fomartDateToString(new Date(), DateUtils.SECONDS_FORMAT));
    	
    	pipeList.add(new TVNotBroadcastModel(
    			StringUtil.isEmpty(tvMap.get("tvname"+urlId)+"")?"":tvMap.get("tvname"+urlId)+"", getTvMapArr(tvMap.get("alias"+urlId)+""), 
				StringUtil.isEmpty(tvMap.get("sets"+urlId)+"")?"":tvMap.get("sets"+urlId)+"", 
				StringUtil.isEmpty(tvMap.get("releasetime"+urlId)+"")?"":tvMap.get("releasetime"+urlId)+"",
				actorArr, actorRoleArr, getTvMapArr(tvMap.get("director"+urlId)+""), getTvMapArr(tvMap.get("writer"+urlId)+""), 
				StringUtil.isEmpty(tvMap.get("area"+urlId)+"")?"":tvMap.get("area"+urlId)+"",
				StringUtil.isEmpty(tvMap.get("language"+urlId)+"")?"":tvMap.get("language"+urlId)+"",
				StringUtil.isEmpty(tvMap.get("introduction"+urlId)+"")?"":tvMap.get("introduction"+urlId)+"",
				StringUtil.isEmpty(tvMap.get("showtype"+urlId)+"")?"":tvMap.get("showtype"+urlId)+"", 
				StringUtil.isEmpty(tvMap.get("timelength"+urlId)+"")?"":tvMap.get("timelength"+urlId)+""));
    	
    	page.putField(TVNotBroadcastModel.class.getName(), pipeList);
    	
}
    
    public String[] getTvMapArr(String tvMap){
		int i=0;
		String[] aliasArr =null;
		String alias = tvMap;
		if(StringUtil.isNotEmpty(alias)){
			String[]  aliasStr = alias.split("/");
			aliasArr =new String[aliasStr.length];
			for(String resAlia : aliasStr) {
				aliasArr[i]=resAlia;
				i++;
			}
		}
			return aliasArr;
	}
    
    //判断字符是否包含中文
    public static boolean isContainChinese(String str) {
        Pattern p = Pattern.compile("[\u4e00-\u9fa5]");
        Matcher m = p.matcher(str);
        if (m.find()) {
            return true;
        }
        return false;
    }
      
    @Override
    protected String getSpiderType()
    {
        return "DouBanTVNotBrd";
    }
    
    public List<JSONObject> getStarname(){
    	 List<JSONObject> jsonList = new ArrayList<JSONObject>();
    	 //获取缓存所有的艺人
    	 Map<String, String> targetMap = redisClient.hgetAll(Constants.DataRedisKey.Star_Info.toString());
	     Set<String> set =targetMap.keySet();
	     //获取已经爬取过得缓存艺人
	     Map<String, String> temp = redisClient.hgetAll(Constants.DataRedisKey.Star_Info_Snap.toString());
	     Object starname = null;
	     int length = 20;
	     //设置每次爬取20 艺人  所有艺人缓存与如果已经爬取过得艺人缓存 比较
	     for(String str:set){
	    	 if(temp.containsKey(str)){
	    		 continue;
	    	 }
	    	 starname=targetMap.get(str);
	    	 //缓存爬取过得艺人
	    	 redisClient.hset(Constants.DataRedisKey.Star_Info_Snap.toString(), str, starname.toString());
	    	 //设置爬取过得艺人失效时间
	    	 int timeout= Integer.parseInt(PropertiesUtil.getProperty(Constants.SpiderConfigKey.STAR_INFO_SNAP_TIMEOUT));
	    	 //如果没有设置失效时间   设置获取的失效时间
	    	 if (redisClient.ttl(Constants.DataRedisKey.Star_Info_Snap.toString()) == -1)
	 			redisClient.expire(Constants.DataRedisKey.Star_Info_Snap.toString(), timeout);
	         JSONObject jsonObj = GsonUtils.fromJson(starname.toString(),
                     JSONObject.class);
             jsonList.add(0, jsonObj);
	         length--;
	         if(length==0){
	             break;
	         }
	     }
	     	return jsonList;
    }
}
